
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import lightgbm as lgbm

from sklearn.model_selection import train_test_split, GridSearchCV
from sklearn.linear_model import LinearRegression, Ridge, SGDRegressor, Lasso, LogisticRegression
from sklearn.svm import LinearSVR
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import GradientBoostingRegressor, AdaBoostRegressor

from sklearn.metrics import log_loss, mean_squared_error
import optuna
import shap


df = pd.read_csv('./input/MiningProcess_Flotation_Plant_Database.csv',
                 decimal=",",
                 parse_dates=["date"],
                 infer_datetime_format=True
                )


df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 737453 entries, 0 to 737452
Data columns (total 24 columns):
 #   Column                        Non-Null Count   Dtype         
---  ------                        --------------   -----         
 0   date                          737453 non-null  datetime64[ns]
 1   % Iron Feed                   737453 non-null  float64       
 2   % Silica Feed                 737453 non-null  float64       
 3   Starch Flow                   737453 non-null  float64       
 4   Amina Flow                    737453 non-null  float64       
 5   Ore Pulp Flow                 737453 non-null  float64       
 6   Ore Pulp pH                   737453 non-null  float64       
 7   Ore Pulp Density              737453 non-null  float64       
 8   Flotation Column 01 Air Flow  737453 non-null  float64       
 9   Flotation Column 02 Air Flow  737453 non-null  float64       
 10  Flotation Column 03 Air Flow  737453 non-null  float64       
 11  Flotation Column 04 Air Flow  737453 non-null  float64       
 12  Flotation Column 05 Air Flow  737453 non-null  float64       
 13  Flotation Column 06 Air Flow  737453 non-null  float64       
 14  Flotation Column 07 Air Flow  737453 non-null  float64       
 15  Flotation Column 01 Level     737453 non-null  float64       
 16  Flotation Column 02 Level     737453 non-null  float64       
 17  Flotation Column 03 Level     737453 non-null  float64       
 18  Flotation Column 04 Level     737453 non-null  float64       
 19  Flotation Column 05 Level     737453 non-null  float64       
 20  Flotation Column 06 Level     737453 non-null  float64       
 21  Flotation Column 07 Level     737453 non-null  float64       
 22  % Iron Concentrate            737453 non-null  float64       
 23  % Silica Concentrate          737453 non-null  float64       
dtypes: datetime64[ns](1), float64(23)
memory usage: 135.0 MB


df['time_date'] = df['date'].dt.date
df['time_o_clock'] = df.date.dt.hour


df['time_date'].value_counts()

2017-06-16    4320
2017-07-18    4320
2017-07-10    4320
2017-07-11    4320
2017-07-12    4320
              ... 
2017-05-23    4320
2017-04-10    4319
2017-03-10    4134
2017-03-29    2160
2017-03-16    1080
Name: time_date, Length: 172, dtype: int64


df['date'].value_counts().plot()

<AxesSubplot:>


df['date'].value_counts()

2017-06-16 15:00:00    180
2017-06-16 16:00:00    180
2017-07-14 18:00:00    180
2017-07-14 19:00:00    180
2017-07-14 20:00:00    180
                      ... 
2017-05-19 11:00:00    180
2017-05-19 12:00:00    180
2017-05-19 13:00:00    180
2017-04-10 00:00:00    179
2017-03-10 01:00:00    174
Name: date, Length: 4097, dtype: int64


df.drop(columns = {'time_date', 'time_o_clock'}, inplace= True)
df.drop(columns = {'date'}, inplace= True)
df.info()

<class 'pandas.core.frame.DataFrame'>
RangeIndex: 737453 entries, 0 to 737452
Data columns (total 23 columns):
 #   Column                        Non-Null Count   Dtype  
---  ------                        --------------   -----  
 0   % Iron Feed                   737453 non-null  float64
 1   % Silica Feed                 737453 non-null  float64
 2   Starch Flow                   737453 non-null  float64
 3   Amina Flow                    737453 non-null  float64
 4   Ore Pulp Flow                 737453 non-null  float64
 5   Ore Pulp pH                   737453 non-null  float64
 6   Ore Pulp Density              737453 non-null  float64
 7   Flotation Column 01 Air Flow  737453 non-null  float64
 8   Flotation Column 02 Air Flow  737453 non-null  float64
 9   Flotation Column 03 Air Flow  737453 non-null  float64
 10  Flotation Column 04 Air Flow  737453 non-null  float64
 11  Flotation Column 05 Air Flow  737453 non-null  float64
 12  Flotation Column 06 Air Flow  737453 non-null  float64
 13  Flotation Column 07 Air Flow  737453 non-null  float64
 14  Flotation Column 01 Level     737453 non-null  float64
 15  Flotation Column 02 Level     737453 non-null  float64
 16  Flotation Column 03 Level     737453 non-null  float64
 17  Flotation Column 04 Level     737453 non-null  float64
 18  Flotation Column 05 Level     737453 non-null  float64
 19  Flotation Column 06 Level     737453 non-null  float64
 20  Flotation Column 07 Level     737453 non-null  float64
 21  % Iron Concentrate            737453 non-null  float64
 22  % Silica Concentrate          737453 non-null  float64
dtypes: float64(23)
memory usage: 129.4 MB


df.describe().T


for item in df.keys():
    df[item].plot(title = item, kind = 'hist', bins = 15)
    plt.show()
    display(df[item].describe())
    print('\n')

count    737453.000000
mean         56.294739
std           5.157744
min          42.740000
25%          52.670000
50%          56.080000
75%          59.720000
max          65.780000
Name: % Iron Feed, dtype: float64

count    737453.000000
mean         14.651716
std           6.807439
min           1.310000
25%           8.940000
50%          13.850000
75%          19.600000
max          33.400000
Name: % Silica Feed, dtype: float64

count    737453.000000
mean       2869.140569
std        1215.203734
min           0.002026
25%        2076.320000
50%        3018.430000
75%        3727.730000
max        6300.230000
Name: Starch Flow, dtype: float64

count    737453.000000
mean        488.144697
std          91.230534
min         241.669000
25%         431.796000
50%         504.393000
75%         553.257000
max         739.538000
Name: Amina Flow, dtype: float64

count    737453.000000
mean        397.578372
std           9.699785
min         376.249000
25%         394.264000
50%         399.249000
75%         402.968000
max         418.641000
Name: Ore Pulp Flow, dtype: float64


target_1, target_2 = df['% Iron Concentrate'].copy(), df['% Silica Concentrate'].copy()
feature = df.iloc[:,0:-2].copy()

print('feature list:')
print(feature.keys())

feature list:
Index(['% Iron Feed', '% Silica Feed', 'Starch Flow', 'Amina Flow',
       'Ore Pulp Flow', 'Ore Pulp pH', 'Ore Pulp Density',
       'Flotation Column 01 Air Flow', 'Flotation Column 02 Air Flow',
       'Flotation Column 03 Air Flow', 'Flotation Column 04 Air Flow',
       'Flotation Column 05 Air Flow', 'Flotation Column 06 Air Flow',
       'Flotation Column 07 Air Flow', 'Flotation Column 01 Level',
       'Flotation Column 02 Level', 'Flotation Column 03 Level',
       'Flotation Column 04 Level', 'Flotation Column 05 Level',
       'Flotation Column 06 Level', 'Flotation Column 07 Level'],
      dtype='object')


#'% Silica Concentrate'作為主要目標(target_2)
print('% Silica Concentrate as training target')

train_X, test_X, train_y, test_y = train_test_split(feature, target_2, test_size = 0.3, random_state = 42)
print('train_X.shape = {},\ntrain_y.shape = {},\ntest_X.shape = {},\ntest_y.shape = {}'.format(train_X.shape, train_y.shape, test_X.shape, test_y.shape))

Model = lgbm.LGBMRegressor()
temp_train_X = train_X.copy()
Model.fit(temp_train_X, train_y)

temp_test_X = test_X.copy()

print('model = ', Model)
print('R2 = {}'.format(Model.score(temp_test_X, test_y)))
    
preds = Model.predict(test_X)
rmse = mean_squared_error(test_y, preds, squared=False)
print('rmse = ', rmse)

% Silica Concentrate as training target
train_X.shape = (516217, 21),
train_y.shape = (516217,),
test_X.shape = (221236, 21),
test_y.shape = (221236,)
model =  LGBMRegressor()
R2 = 0.6019593596513104
rmse =  0.7094539708114015


#'% Silica Concentrate'作為主要目標(target_2)
print('% Silica Concentrate as training target')

model_list = [LinearRegression(), Ridge(), Lasso(), SGDRegressor(), LinearSVR(), DecisionTreeRegressor(random_state = 42), AdaBoostRegressor()]

train_X, test_X, train_y, test_y = train_test_split(feature, target_2, test_size = 0.3, random_state = 42)
print('train_X.shape = {},\ntrain_y.shape = {},\ntest_X.shape = {},\ntest_y.shape = {}\n'.format(train_X.shape, train_y.shape, test_X.shape, test_y.shape))

temp_train_X = train_X.copy()
temp_test_X = test_X.copy()


for model in model_list:
    Model = model
    Model.fit(temp_train_X, train_y)
    print('model = ', Model)
    
    preds = Model.predict(test_X)
    rmse = mean_squared_error(test_y, preds, squared=False)
    print('rmse = ', rmse)
    print('train model R2 score = {}'.format(Model.score(temp_train_X, train_y)))
    print('test model R2  score = {}'.format(Model.score(temp_test_X, test_y)))
    print()

% Silica Concentrate as training target
train_X.shape = (516217, 21),
train_y.shape = (516217,),
test_X.shape = (221236, 21),
test_y.shape = (221236,)

model =  LinearRegression()
rmse =  1.0353356915752685
train model R2 score = 0.15315364088774142
test model R2  score = 0.1523016449154302

model =  Ridge()
rmse =  1.035335809785929
train model R2 score = 0.15315363633378387
test model R2  score = 0.15230145134152162

model =  Lasso()
rmse =  1.0578537035507773
train model R2 score = 0.11469173342813932
test model R2  score = 0.11502665452273975

Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.

model =  SGDRegressor()
rmse =  866483896611780.8
train model R2 score = -5.913891229511937e+29
test model R2  score = -5.937449931490704e+29

Liblinear failed to converge, increase the number of iterations.

model =  LinearSVR()
rmse =  1.5497916258772269
train model R2 score = -0.8841375307561647
test model R2  score = -0.8994400391601547

model =  DecisionTreeRegressor(random_state=42)
rmse =  0.3681321979279572
train model R2 score = 0.999589065128728
test model R2  score = 0.8928267067048725

model =  AdaBoostRegressor()
rmse =  1.0808220167503892
train model R2 score = 0.07869286278963039
test model R2  score = 0.07618005491421187


for max_depth in [3, 5, 10, 15, 20, 25, 30, 40]:
    Model = DecisionTreeRegressor(max_depth = max_depth, random_state = 42)
    Model.fit(temp_train_X, train_y)
    preds = Model.predict(test_X)
    rmse = mean_squared_error(test_y, preds, squared=False)
    print('max_depth = {}'.format(max_depth))
    print('rmse = ', rmse)
    print('train model R2 score = {}'.format(Model.score(temp_train_X, train_y)))
    print('test model R2  score = {}'.format(Model.score(temp_test_X, test_y)))

max_depth = 3
rmse =  1.0367785736484798
train model R2 score = 0.15036528812459116
test model R2  score = 0.14993723099882694
max_depth = 5
rmse =  0.9857475064397497
train model R2 score = 0.23510874231041912
test model R2  score = 0.23155932830336878
max_depth = 10
rmse =  0.8402617435378904
train model R2 score = 0.44983518955308444
test model R2  score = 0.4416479044091902
max_depth = 15
rmse =  0.6515032932349165
train model R2 score = 0.6901786533825639
test model R2  score = 0.6643302544759517
max_depth = 20
rmse =  0.47140214115778206
train model R2 score = 0.8824579593677713
test model R2  score = 0.8242634605087934
max_depth = 25
rmse =  0.3917992126404352
train model R2 score = 0.966063076374883
test model R2  score = 0.8786035221550892
max_depth = 30
rmse =  0.3701920479738158
train model R2 score = 0.9929748056862097
test model R2  score = 0.8916239945373164
max_depth = 40
rmse =  0.36765616597999257
train model R2 score = 0.9994264965212007
test model R2  score = 0.8931036991886555


#pick the specific model
model_to_explain = DecisionTreeRegressor(max_depth = 20, random_state = 42)
model_to_explain.fit(temp_train_X, train_y)

DecisionTreeRegressor(max_depth=20, random_state=42)


#原始驗證用資料筆數達22萬筆，21個特徵
sample = temp_test_X.head(1000)


%%time
# explain part of the predictions in the test set
ex = shap.TreeExplainer(model_to_explain)
shap_values = ex.shap_values(sample)
shap.summary_plot(shap_values, sample, feature_names=temp_train_X.columns, plot_type="bar")

Wall time: 16.1 s


%%time
shap.summary_plot(shap_values, sample, feature_names=temp_train_X.columns)

Wall time: 464 ms


shap.dependence_plot("Flotation Column 03 Air Flow", shap_values, sample, interaction_index = "Amina Flow")
shap.dependence_plot("Flotation Column 03 Air Flow", shap_values, sample, interaction_index = "% Silica Feed")
shap.dependence_plot("Amina Flow", shap_values, sample, interaction_index = "% Silica Feed")


tree_explainer = shap.TreeExplainer(Model)


sample_row = 5
data_for_prediction = temp_test_X.iloc[sample_row]
print('data_for_prediction: \n{}'.format(data_for_prediction))
shap_values = tree_explainer.shap_values(data_for_prediction)
print('true y = {}\n'.format(test_y.iloc[sample_row]))
# shap.initjs()
shap.force_plot(tree_explainer.expected_value, shap_values, data_for_prediction, matplotlib = True)

data_for_prediction: 
% Iron Feed                       64.03000
% Silica Feed                      6.26000
Starch Flow                     3284.79000
Amina Flow                       384.12200
Ore Pulp Flow                    401.30000
Ore Pulp pH                       10.25790
Ore Pulp Density                   1.60925
Flotation Column 01 Air Flow     299.49600
Flotation Column 02 Air Flow     304.26100
Flotation Column 03 Air Flow     298.98000
Flotation Column 04 Air Flow     299.85100
Flotation Column 05 Air Flow     301.98000
Flotation Column 06 Air Flow     299.92700
Flotation Column 07 Air Flow     296.69800
Flotation Column 01 Level        607.40000
Flotation Column 02 Level        602.93900
Flotation Column 03 Level        602.84400
Flotation Column 04 Level        503.86100
Flotation Column 05 Level        491.95400
Flotation Column 06 Level        507.45600
Flotation Column 07 Level        493.73800
Name: 292500, dtype: float64
true y = 1.18


sample_row = 222
data_for_prediction = temp_train_X.iloc[sample_row]
print('data_for_prediction: \n{}'.format(data_for_prediction))
shap_values = tree_explainer.shap_values(data_for_prediction)
print('true y = {}\n'.format(test_y.iloc[sample_row]))
# shap.initjs()
shap.force_plot(tree_explainer.expected_value, shap_values, data_for_prediction, matplotlib = True)

data_for_prediction: 
% Iron Feed                       48.810000
% Silica Feed                     25.310000
Starch Flow                     3480.870000
Amina Flow                       459.649000
Ore Pulp Flow                    380.273000
Ore Pulp pH                       10.041200
Ore Pulp Density                   1.692440
Flotation Column 01 Air Flow     299.907000
Flotation Column 02 Air Flow     250.450000
Flotation Column 03 Air Flow     297.870000
Flotation Column 04 Air Flow     300.547000
Flotation Column 05 Air Flow     300.642733
Flotation Column 06 Air Flow     299.451000
Flotation Column 07 Air Flow     248.313000
Flotation Column 01 Level        554.575000
Flotation Column 02 Level        553.162000
Flotation Column 03 Level        559.022000
Flotation Column 04 Level        518.639000
Flotation Column 05 Level        562.547000
Flotation Column 06 Level        557.203000
Flotation Column 07 Level        573.671000
Name: 616331, dtype: float64
true y = 4.1582801664


#重現
Model = DecisionTreeRegressor(max_depth = 20, random_state = 42)
Model.fit(temp_train_X, train_y)
    
print('model = ', Model)
    
preds = Model.predict(test_X)
rmse = mean_squared_error(test_y, preds, squared=False)
print('rmse = ', rmse)
print('train model R2 score = {}'.format(Model.score(temp_train_X, train_y)))
print('test model R2  score = {}'.format(Model.score(temp_test_X, test_y)))
print('params = {}'.format(Model.get_params()))
print('depth = {}'.format(Model.get_depth()))
print('nth_leaves = {}'.format(Model.get_n_leaves()))

model =  DecisionTreeRegressor(max_depth=20, random_state=42)
rmse =  0.47140214115778206
train model R2 score = 0.8824579593677713
test model R2  score = 0.8242634605087934
params = {'ccp_alpha': 0.0, 'criterion': 'mse', 'max_depth': 20, 'max_features': None, 'max_leaf_nodes': None, 'min_impurity_decrease': 0.0, 'min_impurity_split': None, 'min_samples_leaf': 1, 'min_samples_split': 2, 'min_weight_fraction_leaf': 0.0, 'random_state': 42, 'splitter': 'best'}
depth = 20
nth_leaves = 35058


%%time
#using the GridSearch function from sklearn 
parameters = {'max_depth': [12, 14, 16, 18, 20],
              'min_samples_split': [2, 10, 20, 40, 60, 80, 100],
              'max_features': ('auto', 'sqrt', 'log2'),
              'splitter': ('best', 'random')} 

Model = DecisionTreeRegressor(random_state = 42)
reg = GridSearchCV(Model, parameters)
reg.fit(temp_train_X, train_y)

Wall time: 44min 8s

GridSearchCV(estimator=DecisionTreeRegressor(random_state=42),
             param_grid={'max_depth': [12, 14, 16, 18, 20],
                         'max_features': ('auto', 'sqrt', 'log2'),
                         'min_samples_split': [2, 10, 20, 40, 60, 80, 100],
                         'splitter': ('best', 'random')})


print(reg.best_params_)
print(reg.best_score_)

{'max_depth': 20, 'max_features': 'auto', 'min_samples_split': 10, 'splitter': 'best'}
0.8218425725981036


print('train model R2 score = {}'.format(reg.score(temp_train_X, train_y)))
print('test model R2  score = {}'.format(reg.score(temp_test_X, test_y)))

train model R2 score = 0.8792676090278375
test model R2  score = 0.824697249452386


def objective(trial, data = feature, target = target_2):
    train_X, test_X, train_y, test_y = train_test_split(data, target, test_size = 0.3, random_state = 42)

    temp_train_X = train_X.copy()  
    param = {
        'max_depth': trial.suggest_int('max_depth', 12, 20, step = 1),
        'min_samples_split': trial.suggest_int('min_samples_split', 2, 100, step = 1),
        'max_features': trial.suggest_categorical('max_features', ['auto', 'sqrt', 'log2']),
        'splitter': trial.suggest_categorical('splitter', ['best', 'random'])
    }
    
    temp_test_X = test_X.copy()
    model = DecisionTreeRegressor(random_state = 42, **param )
    model.fit(temp_train_X, train_y)
    preds = model.predict(temp_test_X)
    
    rmse = mean_squared_error(test_y, preds,squared = False)
    
    return rmse


%%time
study = optuna.create_study(direction='minimize')
study.optimize(objective, n_trials=40)
print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params)

[I 2022-07-10 14:08:59,093] A new study created in memory with name: no-name-91f157a9-6131-4c9b-bfdb-af19a68588eb
[I 2022-07-10 14:08:59,712] Trial 0 finished with value: 0.8953496164895249 and parameters: {'max_depth': 15, 'min_samples_split': 91, 'max_features': 'log2', 'splitter': 'random'}. Best is trial 0 with value: 0.8953496164895249.
[I 2022-07-10 14:09:10,702] Trial 1 finished with value: 0.6967632873373096 and parameters: {'max_depth': 14, 'min_samples_split': 78, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 1 with value: 0.6967632873373096.
[I 2022-07-10 14:09:11,332] Trial 2 finished with value: 0.8652199133807803 and parameters: {'max_depth': 16, 'min_samples_split': 48, 'max_features': 'sqrt', 'splitter': 'random'}. Best is trial 1 with value: 0.6967632873373096.
[I 2022-07-10 14:09:11,967] Trial 3 finished with value: 0.860933773541887 and parameters: {'max_depth': 17, 'min_samples_split': 21, 'max_features': 'log2', 'splitter': 'random'}. Best is trial 1 with value: 0.6967632873373096.
[I 2022-07-10 14:09:14,036] Trial 4 finished with value: 0.6167669547561885 and parameters: {'max_depth': 19, 'min_samples_split': 20, 'max_features': 'auto', 'splitter': 'random'}. Best is trial 4 with value: 0.6167669547561885.
[I 2022-07-10 14:09:14,692] Trial 5 finished with value: 0.8667649278342843 and parameters: {'max_depth': 17, 'min_samples_split': 57, 'max_features': 'sqrt', 'splitter': 'random'}. Best is trial 4 with value: 0.6167669547561885.
[I 2022-07-10 14:09:27,599] Trial 6 finished with value: 0.5397192046531641 and parameters: {'max_depth': 18, 'min_samples_split': 50, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 6 with value: 0.5397192046531641.
[I 2022-07-10 14:09:29,279] Trial 7 finished with value: 0.7456059848349866 and parameters: {'max_depth': 14, 'min_samples_split': 77, 'max_features': 'auto', 'splitter': 'random'}. Best is trial 6 with value: 0.5397192046531641.
[I 2022-07-10 14:09:30,885] Trial 8 finished with value: 0.7646489191292777 and parameters: {'max_depth': 13, 'min_samples_split': 55, 'max_features': 'auto', 'splitter': 'random'}. Best is trial 6 with value: 0.5397192046531641.
[I 2022-07-10 14:09:33,658] Trial 9 finished with value: 0.6438594373997173 and parameters: {'max_depth': 18, 'min_samples_split': 58, 'max_features': 'log2', 'splitter': 'best'}. Best is trial 6 with value: 0.5397192046531641.
[I 2022-07-10 14:09:47,308] Trial 10 finished with value: 0.4771236110436543 and parameters: {'max_depth': 20, 'min_samples_split': 34, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 10 with value: 0.4771236110436543.
[I 2022-07-10 14:10:00,970] Trial 11 finished with value: 0.47810794031573506 and parameters: {'max_depth': 20, 'min_samples_split': 35, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 10 with value: 0.4771236110436543.
[I 2022-07-10 14:10:14,696] Trial 12 finished with value: 0.4704547736203571 and parameters: {'max_depth': 20, 'min_samples_split': 7, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:10:28,515] Trial 13 finished with value: 0.47140214115778206 and parameters: {'max_depth': 20, 'min_samples_split': 2, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:10:42,231] Trial 14 finished with value: 0.47140214115778206 and parameters: {'max_depth': 20, 'min_samples_split': 2, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:10:45,253] Trial 15 finished with value: 0.5975498637594137 and parameters: {'max_depth': 19, 'min_samples_split': 2, 'max_features': 'sqrt', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:10:54,762] Trial 16 finished with value: 0.7715788544913298 and parameters: {'max_depth': 12, 'min_samples_split': 17, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:11:08,082] Trial 17 finished with value: 0.5024123539955824 and parameters: {'max_depth': 19, 'min_samples_split': 11, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:11:10,895] Trial 18 finished with value: 0.6450086620859322 and parameters: {'max_depth': 18, 'min_samples_split': 30, 'max_features': 'sqrt', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:11:13,486] Trial 19 finished with value: 0.670542679943978 and parameters: {'max_depth': 16, 'min_samples_split': 10, 'max_features': 'log2', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:11:27,132] Trial 20 finished with value: 0.4739974475795149 and parameters: {'max_depth': 20, 'min_samples_split': 26, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:11:40,860] Trial 21 finished with value: 0.4715835811837914 and parameters: {'max_depth': 20, 'min_samples_split': 4, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:11:54,177] Trial 22 finished with value: 0.5024123539955824 and parameters: {'max_depth': 19, 'min_samples_split': 11, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:12:07,966] Trial 23 finished with value: 0.47140214115778206 and parameters: {'max_depth': 20, 'min_samples_split': 2, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:12:21,410] Trial 24 finished with value: 0.5091377748701501 and parameters: {'max_depth': 19, 'min_samples_split': 43, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:12:34,348] Trial 25 finished with value: 0.5332008386497631 and parameters: {'max_depth': 18, 'min_samples_split': 14, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:12:48,070] Trial 26 finished with value: 0.4739578239691058 and parameters: {'max_depth': 20, 'min_samples_split': 24, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:12:50,804] Trial 27 finished with value: 0.6639576379148449 and parameters: {'max_depth': 17, 'min_samples_split': 7, 'max_features': 'log2', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:12:53,755] Trial 28 finished with value: 0.5957559197431085 and parameters: {'max_depth': 19, 'min_samples_split': 41, 'max_features': 'sqrt', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:13:05,528] Trial 29 finished with value: 0.617701809694936 and parameters: {'max_depth': 16, 'min_samples_split': 85, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:13:08,034] Trial 30 finished with value: 0.7466106033083885 and parameters: {'max_depth': 15, 'min_samples_split': 98, 'max_features': 'log2', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:13:21,919] Trial 31 finished with value: 0.4716351717186849 and parameters: {'max_depth': 20, 'min_samples_split': 3, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:13:35,735] Trial 32 finished with value: 0.47140214115778206 and parameters: {'max_depth': 20, 'min_samples_split': 2, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:13:49,087] Trial 33 finished with value: 0.5031778754167713 and parameters: {'max_depth': 19, 'min_samples_split': 16, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:14:02,821] Trial 34 finished with value: 0.47117645173317796 and parameters: {'max_depth': 20, 'min_samples_split': 8, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:14:04,807] Trial 35 finished with value: 0.6392761354607346 and parameters: {'max_depth': 18, 'min_samples_split': 70, 'max_features': 'auto', 'splitter': 'random'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:14:18,224] Trial 36 finished with value: 0.5033488733292256 and parameters: {'max_depth': 19, 'min_samples_split': 20, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:14:18,974] Trial 37 finished with value: 0.8007926555718241 and parameters: {'max_depth': 20, 'min_samples_split': 8, 'max_features': 'sqrt', 'splitter': 'random'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:14:32,333] Trial 38 finished with value: 0.5029333546889279 and parameters: {'max_depth': 19, 'min_samples_split': 15, 'max_features': 'auto', 'splitter': 'best'}. Best is trial 12 with value: 0.4704547736203571.
[I 2022-07-10 14:14:34,265] Trial 39 finished with value: 0.6579226195426993 and parameters: {'max_depth': 17, 'min_samples_split': 25, 'max_features': 'auto', 'splitter': 'random'}. Best is trial 12 with value: 0.4704547736203571.

Number of finished trials: 40
Best trial: {'max_depth': 20, 'min_samples_split': 7, 'max_features': 'auto', 'splitter': 'best'}
Wall time: 5min 35s


study.best_trial

FrozenTrial(number=12, values=[0.4704547736203571], datetime_start=datetime.datetime(2022, 7, 10, 14, 10, 0, 970425), datetime_complete=datetime.datetime(2022, 7, 10, 14, 10, 14, 696394), params={'max_depth': 20, 'min_samples_split': 7, 'max_features': 'auto', 'splitter': 'best'}, distributions={'max_depth': IntUniformDistribution(high=20, low=12, step=1), 'min_samples_split': IntUniformDistribution(high=100, low=2, step=1), 'max_features': CategoricalDistribution(choices=('auto', 'sqrt', 'log2')), 'splitter': CategoricalDistribution(choices=('best', 'random'))}, user_attrs={}, system_attrs={}, intermediate_values={}, trial_id=12, state=TrialState.COMPLETE, value=None)


Model = DecisionTreeRegressor(max_depth = 20, min_samples_split = 7, max_features = 'auto', splitter = 'best', random_state = 42)
Model.fit(temp_train_X, train_y)
    
print('model = ', Model)
    
preds = Model.predict(test_X)
rmse = mean_squared_error(test_y, preds, squared=False)
print('rmse = ', rmse)
print('train model R2 score = {}'.format(Model.score(temp_train_X, train_y)))
print('test model R2  score = {}'.format(Model.score(temp_test_X, test_y)))
print('params = {}'.format(Model.get_params()))
print('depth = {}'.format(Model.get_depth()))
print('nth_leaves = {}'.format(Model.get_n_leaves()))

model =  DecisionTreeRegressor(max_depth=20, max_features='auto', min_samples_split=7,
                      random_state=42)
rmse =  0.4704547736203571
train model R2 score = 0.880790093778892
test model R2  score = 0.8249690992203196
params = {'ccp_alpha': 0.0, 'criterion': 'mse', 'max_depth': 20, 'max_features': 'auto', 'max_leaf_nodes': None, 'min_impurity_decrease': 0.0, 'min_impurity_split': None, 'min_samples_leaf': 1, 'min_samples_split': 7, 'min_weight_fraction_leaf': 0.0, 'random_state': 42, 'splitter': 'best'}
depth = 20
nth_leaves = 21846


study.trials_dataframe()


optuna.visualization.plot_optimization_history(study)


optuna.visualization.plot_parallel_coordinate(study)


optuna.visualization.plot_slice(study)


#Visualize parameter importances.
optuna.visualization.plot_param_importances(study)

Mining Process: Machine Learning and Analysis¶

Import Library & Dataset¶

EDA¶

Modelling¶

Model Baseline¶

Other model¶

Explanation¶

Explanation for model¶

Explanation for prediction¶

Hyperparameter Tuning¶

Sklearn¶

Optuna¶

Conclusion¶

	count	mean	std	min	25%	50%	75%	max
% Iron Feed	737453.0	56.294739	5.157744	42.740000	52.670	56.080000	59.720	65.780
% Silica Feed	737453.0	14.651716	6.807439	1.310000	8.940	13.850000	19.600	33.400
Starch Flow	737453.0	2869.140569	1215.203734	0.002026	2076.320	3018.430000	3727.730	6300.230
Amina Flow	737453.0	488.144697	91.230534	241.669000	431.796	504.393000	553.257	739.538
Ore Pulp Flow	737453.0	397.578372	9.699785	376.249000	394.264	399.249000	402.968	418.641
...	...	...	...	...	...	...	...	...
Flotation Column 05 Level	737453.0	425.251706	84.535822	166.991000	357.653	408.773000	484.329	675.644
Flotation Column 06 Level	737453.0	429.941018	89.862225	155.841000	358.497	424.664575	492.684	698.861
Flotation Column 07 Level	737453.0	421.021231	84.891491	175.349000	356.772	411.065000	476.465	659.902
% Iron Concentrate	737453.0	65.050068	1.118645	62.050000	64.370	65.210000	65.860	68.010
% Silica Concentrate	737453.0	2.326763	1.125554	0.600000	1.440	2.000000	3.010	5.530

	number	value	datetime_start	datetime_complete	duration	params_max_depth	params_max_features	params_min_samples_split	params_splitter	state
0	0	0.895350	2022-07-10 14:08:59.095030	2022-07-10 14:08:59.712766	0 days 00:00:00.617736	15	log2	91	random	COMPLETE
1	1	0.696763	2022-07-10 14:08:59.712766	2022-07-10 14:09:10.702216	0 days 00:00:10.989450	14	auto	78	best	COMPLETE
2	2	0.865220	2022-07-10 14:09:10.717809	2022-07-10 14:09:11.332133	0 days 00:00:00.614324	16	sqrt	48	random	COMPLETE
3	3	0.860934	2022-07-10 14:09:11.332133	2022-07-10 14:09:11.967950	0 days 00:00:00.635817	17	log2	21	random	COMPLETE
4	4	0.616767	2022-07-10 14:09:11.967950	2022-07-10 14:09:14.036189	0 days 00:00:02.068239	19	auto	20	random	COMPLETE
...	...	...	...	...	...	...	...	...	...	...
35	35	0.639276	2022-07-10 14:14:02.821498	2022-07-10 14:14:04.807781	0 days 00:00:01.986283	18	auto	70	random	COMPLETE
36	36	0.503349	2022-07-10 14:14:04.807781	2022-07-10 14:14:18.224468	0 days 00:00:13.416687	19	auto	20	best	COMPLETE
37	37	0.800793	2022-07-10 14:14:18.224468	2022-07-10 14:14:18.974954	0 days 00:00:00.750486	20	sqrt	8	random	COMPLETE
38	38	0.502933	2022-07-10 14:14:18.974954	2022-07-10 14:14:32.333737	0 days 00:00:13.358783	19	auto	15	best	COMPLETE
39	39	0.657923	2022-07-10 14:14:32.333737	2022-07-10 14:14:34.265480	0 days 00:00:01.931743	17	auto	25	random	COMPLETE